package fetcher

import (
    "bufio"
    "fmt"
    "golang.org/x/net/html/charset"
    "golang.org/x/text/encoding"
    "golang.org/x/text/encoding/unicode"
    "golang.org/x/text/transform"
        "io/ioutil"
    "log"
    "net/http"
    "time"
)

// 定义一个每隔100毫秒自动填充时间数据的的channel
var rateLimiter = time.Tick(10 * time.Millisecond)

func Fetch(url string) ([]byte, error) {
    // 格式:2018-08-04 17:58:29.4974511 +0800 CST m=+1.113025101
    // fmt.Println(<-rateLimiter)
    
    <-rateLimiter
    
    reponse, err := http.Get(url)
    if err != nil {
        panic(err)
    }
    
    defer reponse.Body.Close()
    
    if reponse.StatusCode != http.StatusOK {
        return nil, fmt.Errorf("error: status code not OK,wrong status code: %d", reponse.StatusCode)
    }
    
    // utf6Reader := transform.NewReader(reponse.Body, simplifiedchinese.GBK.NewDecoder())
    
    bodyReader := bufio.NewReader(reponse.Body)
    
    analysisEncoding := determineEncoding(bodyReader)
    
    newReader := transform.NewReader(reponse.Body, analysisEncoding.NewDecoder())
    
    return ioutil.ReadAll(newReader)
}

// 根据reader分析编码
func determineEncoding(r *bufio.Reader) encoding.Encoding {
    bytes, err := bufio.NewReader(r).Peek(1024)
    if err != nil {
        log.Printf("Fetcher error: %v", err)
        return unicode.UTF8
    }
    enco, _, _ := charset.DetermineEncoding(bytes, "")
    return enco
}
